package spider;

import java.util.*;

/**
 * 爬虫主程序
 */
public class SpiderMain {

    /**
     * 网站根目录
     */
    private String baseURL;

    /**
     * 待爬取的url集合
     */
    private Set<String> urlSet = new LinkedHashSet<>();

    /**
     * 已经爬取完成的url集合
     */
    private Set<String> finishedSet = new HashSet<>();

    /**
     * 爬取的页面的采集器
     */
    private Capturer urlCapturer;
    /**
     * 爬取的数据的采集器
     */
    private Capturer dataCapturer;
    /**
     * 数据存储对象
     */
    private DataDao dao;

    public SpiderMain(String baseURL, Capturer urlCapturer, Capturer dataCapturer, DataDao dao) {
        this.baseURL = baseURL;
        this.urlCapturer = urlCapturer;
        this.dataCapturer = dataCapturer;
        this.dao = dao;
        this.urlSet.add(baseURL);
    }

    public void start() {

        while (urlSet.isEmpty() == false) {
            String url = urlSet.iterator().next();
            String fullUrl = url;
            if (fullUrl.startsWith("http") == false) {
                // 提取baseURL非参数部分 ==> 地址部分
                fullUrl = baseURL.replaceAll("\\?.*", "") + "/" + url;
            }

            // 采集 URL
            if (urlCapturer != null) {
                Set<String> newUrlSet = urlCapturer.capture(fullUrl);
                if (newUrlSet.isEmpty() == false) {
                    for (String newUrl : newUrlSet) {
                        if (urlSet.contains(newUrl) || finishedSet.contains(newUrl)) {
                            continue;
                        }
                        urlSet.add(newUrl);
                    }
                }
            }
            // 采集数据 并保存
            save(dataCapturer.capture(fullUrl), fullUrl);

            // url 转移到 finishedSet
            finishedSet.add(url);
            urlSet.remove(url);
        }
    }

    /**
     * 保存数据
     *
     * @param data
     * @param <T>
     */
    public <T> void save(T data, String url) {
        if (data instanceof Map) {
            Map<String, String> map = (Map<String, String>) data;
            map.put("url", url);
        } else if (data instanceof List) {
            List<String> list = (List<String>) data;
            if (list.isEmpty()) return;
        } else {
            // 如果是实体类, 那么通过java反射, 将url设置到 data 中, 前提data有url属性
        }
        dao.save(data);
    }

    public Set<String> getUrlSet() {
        return urlSet;
    }
}
